package ar.edu.unlu.sistemas.ws.test;

import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.HashMap;
import java.util.Map;

public class Indexer {

	protected static Map<String,Long> tfs;
	
	/**
	 * @param args
	 */
	public static void main(String[] args) throws Exception{
		File dir = new File("/home/pepo/Dropbox/Tesis/Corpus/El-Universal-MX");
		File[] files = dir.listFiles();
		tfs = new HashMap<String,Long>();
		for (int i = 0; i < files.length; i++) {
			System.out.println(i+"/"+files.length);
			createIndexFromFile(new FileReader(files[i]));
		}
		dir = new File("/home/pepo/Escritorio");
		File temp = File.createTempFile("zipf", ".txt", dir);
		DataOutputStream dos = new DataOutputStream(new FileOutputStream(temp));
		System.out.println("Escribiendo archivo..");
		for (String word : tfs.keySet()) {
			dos.write((tfs.get(word).toString()+"|").getBytes("UTF-8"));
		}
		dos.close();
	}
	
	public static String[] tokenize(String line) throws UnsupportedEncodingException{
		 line = line.toLowerCase();
		 line = line.replaceAll("[óÓòÒôôöÖõÕ]", "o");
		 line = line.replaceAll("[íÍìÌîÎïÏ]", "i");
		 line = line.replaceAll("[áÁàÀâÂÄåÅãÃä]", "a");
		 line = line.replaceAll("[éÉèÈÊêëË]", "e");
		 line = line.replaceAll("[úüÚÜùÙûÛ]", "u");
		 line = line.replaceAll("ñ", "n");
		 line = line.replaceAll("\\W", " ");
		 line = line.replaceAll("[1234567890]"," ");
		 return line.split("\\s+");
	}
	
	/**
	 * Método principal de la clase indexer. A partir de un archivo de texto, se lo indexa en la base
	 * de datos mediante la representación del modelo vectorial.
	 * 
	 * @param file
	 * @return Un diccionario con la key que es el término y el valor el objeto q lo representa
	 * @throws FileNotFoundException
	 * @throws IOException
	 * @throws UnsupportedEncodingException
	 */
	public static void createIndexFromFile(FileReader file) throws FileNotFoundException,IOException, UnsupportedEncodingException {
		String line;
		Long l;
		BufferedReader in = new BufferedReader(file);
			while ((line = in.readLine()) != null) {
				String[] tokens = tokenize(line);
				for (int i=0;i<tokens.length;i++) {
					if (tfs.containsKey(tokens[i])) {
						l = tfs.get(tokens[i]);
						tfs.put(tokens[i],l+1);
					} else {
						tfs.put(tokens[i], new Long("1"));
					}
				}
			}
	}

}
